# -*- coding: utf-8 -*- 
# @Time : 2021/4/4 15:43 
# @Author : Dong 
# @File : myself_proxy_pool.py
import time
from multiprocessing import Pool

import requests
import myself_proxy_pool_constant
from pyquery import PyQuery as pq

# 1.拿到需要代理的ip
class Download():
    def __init__(self):
        self.ip_list = []
        self.url = myself_proxy_pool_constant.url
        self.headers = myself_proxy_pool_constant.headers

    def get(self,num):
        response = requests.get(self.url + str(num))
        if response.status_code == 200:
            response = response.text
            print('正在抓取第'+ str(num)+'页')
            self.range_num(response)
        else:
            print('请求失败')

    def parse_one(self,html):
        doc = pq(html)
        items = doc('#list > table > tbody > tr').items()
        for item in items:
            list_data = item('td').text()
            f_l = list_data.split(' ')
            yield {
                f_l[3]: f_l[0] + ':' + f_l[1],
                # 'location': f_l[4],
                # 'last_time': f_l[7] + ' ' + f_l[8]
            }

    def check_ip(self,proxies):
        try:
            response = requests.get('https://www.baidu.com',headers=self.headers,proxies=proxies,timeout=0.1)
            if response.status_code == 200:
                print('ip有效',proxies)
                self.ip_list.append(proxies)
        except Exception as e:
            print(e)

    def range_num(self,response):
        for i in self.parse_one(response):
            self.check_ip(i)


if __name__ == '__main__':
    proxy = Download()
    for i in range(1,11):
        time.sleep(1)
        proxy.get(i)
    print(len(proxy.ip_list))
    # pool = Pool()
    # pool.map(main, [i for i in range(10)])



